package com.zju.cst.spider;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.zju.cst.spider.domain.News;
import com.zju.cst.spider.mapper.NewsMapper;
import com.zju.cst.spider.utils.DBUtils;
import com.zju.cst.spider.utils.StreamUtils;
import com.zju.cst.spider.utils.URLUtils;

/**
 * 爬虫入口程序，count参数控制爬取层数，默认为3层，爬取网易新闻时大概200+数据。
 * @author Best_
 */
public class Main {
	
	private static Map<String, String> params = new HashMap<String, String>();
	static{
		params.put("体育", "http://sports.people.com.cn/");
		params.put("财经", "http://finance.people.com.cn/");
		params.put("旅游", "http://travel.people.com.cn/");
		params.put("时政", "http://politics.people.com.cn/");
	}
	public static void main(String[] args) throws Exception {
		
		for(String key : params.keySet()){
			//int count = 30;
			String url = params.get(key);
			SqlSession session = null;
			try {
				session = DBUtils.getSession();
				NewsMapper mapper = session.getMapper(NewsMapper.class);
				String content = URLUtils.getContent(url);
				List<String> regexUrls = URLUtils.regex(content, "n1/\\d{4}/\\d{4}(.*?)html");
				for(String regexUrl : regexUrls){
					try {
						regexUrl = url + regexUrl;
						News news = new News();
						
						String targetContent = URLUtils.getContent(regexUrl);
						news.setContent(targetContent);
						Document root = Jsoup.parse(targetContent);
						String text = Parse.getText(root);
						news.setText(text);
						String title = Parse.getTitle(root);
						news.setTitle(title);
						news.setBaseUrl(regexUrl);
						news.setClassify(key);
						//将数据存入数据库中，并指定分类
						mapper.insert(news);
						//count--;
						//if(count < 0) break;
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
				session.commit();
			} catch (Exception e) {
				e.printStackTrace();
				if (session != null) session.rollback();
			} finally {
				StreamUtils.close(session);
			}
		}
	}

}
